{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2e8f4f85",
   "metadata": {},
   "source": [
    "# 机器学习练习-集成学习\n",
    "代码更新地址：https://github.com/fengdu78/WZU-machine-learning-course\n",
    "\n",
    "\n",
    "代码修改并注释：黄海广，haiguang2000@wzu.edu.cn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3257f67d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc304344",
   "metadata": {},
   "source": [
    "## 生成数据\n",
    "生成12000行的数据，训练集和测试集按照3:1划分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6ebd8121",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_hastie_10_2\n",
    "\n",
    "data, target = make_hastie_10_2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c6dfeb30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((9000, 10), (3000, 10))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, target, random_state=123)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "579ac5c7",
   "metadata": {},
   "source": [
    "## 模型对比\n",
    "对比六大模型，都使用默认参数，因为数据是"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f808ac16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.47488889 (+/- 0.00),耗时0.04秒。模型名称[Logistic Regression]\n",
      "Accuracy: 0.88966667 (+/- 0.01),耗时16.34秒。模型名称[Random Forest]\n",
      "Accuracy: 0.88311111 (+/- 0.00),耗时3.39秒。模型名称[AdaBoost]\n",
      "Accuracy: 0.91388889 (+/- 0.01),耗时13.14秒。模型名称[GBDT]\n",
      "[20:15:54] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[20:15:55] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[20:15:56] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[20:15:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "[20:15:57] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy: 0.92977778 (+/- 0.00),耗时3.60秒。模型名称[XGBoost]\n",
      "Accuracy: 0.93188889 (+/- 0.01),耗时0.58秒。模型名称[LightGBM]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "import time\n",
    "\n",
    "clf1 = LogisticRegression()\n",
    "clf2 = RandomForestClassifier()\n",
    "clf3 = AdaBoostClassifier()\n",
    "clf4 = GradientBoostingClassifier()\n",
    "clf5 = XGBClassifier()\n",
    "clf6 = LGBMClassifier()\n",
    "\n",
    "for clf, label in zip([clf1, clf2, clf3, clf4, clf5, clf6], [\n",
    "        'Logistic Regression', 'Random Forest', 'AdaBoost', 'GBDT', 'XGBoost',\n",
    "        'LightGBM'\n",
    "]):\n",
    "    start = time.time()\n",
    "    scores = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=5)\n",
    "    end = time.time()\n",
    "    running_time = end - start\n",
    "    print(\"Accuracy: %0.8f (+/- %0.2f),耗时%0.2f秒。模型名称[%s]\" %\n",
    "          (scores.mean(), scores.std(), running_time, label))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caa60220",
   "metadata": {},
   "source": [
    "对比了六大模型，可以看出，逻辑回归速度最快，但准确率最低。\n",
    "而LightGBM，速度快，而且准确率最高，所以，现在处理结构化数据的时候，大部分都是用LightGBM算法。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d63e7313",
   "metadata": {},
   "source": [
    "## XGBoost的使用"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ef5b733",
   "metadata": {},
   "source": [
    "### 1.原生XGBoost的使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "96d20084",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "#记录程序运行时间\n",
    "import time\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "#xgb矩阵赋值\n",
    "xgb_train = xgb.DMatrix(X_train, y_train)\n",
    "xgb_test = xgb.DMatrix(X_test, label=y_test)\n",
    "##参数\n",
    "params = {\n",
    "    'booster': 'gbtree',\n",
    "#     'silent': 1,  #设置成1则没有运行信息输出，最好是设置为0.\n",
    "    #'nthread':7,# cpu 线程数 默认最大\n",
    "    'eta': 0.007,  # 如同学习率\n",
    "    'min_child_weight': 3,\n",
    "    # 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言\n",
    "    #，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。\n",
    "    #这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。\n",
    "    'max_depth': 6,  # 构建树的深度，越大越容易过拟合\n",
    "    'gamma': 0.1,  # 树的叶子节点上作进一步分区所需的最小损失减少,越大越保守，一般0.1、0.2这样子。\n",
    "    'subsample': 0.7,  # 随机采样训练样本\n",
    "    'colsample_bytree': 0.7,  # 生成树时进行的列采样 \n",
    "    'lambda': 2,  # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。\n",
    "    #'alpha':0, # L1 正则项参数\n",
    "    #'scale_pos_weight':1, #如果取值大于0的话，在类别样本不平衡的情况下有助于快速收敛。\n",
    "    #'objective': 'multi:softmax', #多分类的问题\n",
    "    #'num_class':10, # 类别数，多分类与 multisoftmax 并用\n",
    "    'seed': 1000,  #随机种子\n",
    "    #'eval_metric': 'auc'\n",
    "}\n",
    "plst = list(params.items())\n",
    "num_rounds = 500  # 迭代次数\n",
    "watchlist = [(xgb_train, 'train'), (xgb_test, 'val')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7421bd6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-rmse:1.11000\tval-rmse:1.10422\n",
      "[1]\ttrain-rmse:1.10734\tval-rmse:1.10182\n",
      "[2]\ttrain-rmse:1.10465\tval-rmse:1.09932\n",
      "[3]\ttrain-rmse:1.10207\tval-rmse:1.09694\n",
      "[4]\ttrain-rmse:1.09944\tval-rmse:1.09453\n",
      "[5]\ttrain-rmse:1.09682\tval-rmse:1.09211\n",
      "[6]\ttrain-rmse:1.09424\tval-rmse:1.08975\n",
      "[7]\ttrain-rmse:1.09175\tval-rmse:1.08745\n",
      "[8]\ttrain-rmse:1.08923\tval-rmse:1.08511\n",
      "[9]\ttrain-rmse:1.08664\tval-rmse:1.08275\n",
      "[10]\ttrain-rmse:1.08410\tval-rmse:1.08039\n",
      "[11]\ttrain-rmse:1.08167\tval-rmse:1.07811\n",
      "[12]\ttrain-rmse:1.07922\tval-rmse:1.07581\n",
      "[13]\ttrain-rmse:1.07675\tval-rmse:1.07353\n",
      "[14]\ttrain-rmse:1.07434\tval-rmse:1.07129\n",
      "[15]\ttrain-rmse:1.07190\tval-rmse:1.06903\n",
      "[16]\ttrain-rmse:1.06943\tval-rmse:1.06677\n",
      "[17]\ttrain-rmse:1.06713\tval-rmse:1.06467\n",
      "[18]\ttrain-rmse:1.06476\tval-rmse:1.06252\n",
      "[19]\ttrain-rmse:1.06244\tval-rmse:1.06040\n",
      "[20]\ttrain-rmse:1.06015\tval-rmse:1.05831\n",
      "[21]\ttrain-rmse:1.05791\tval-rmse:1.05630\n",
      "[22]\ttrain-rmse:1.05558\tval-rmse:1.05421\n",
      "[23]\ttrain-rmse:1.05328\tval-rmse:1.05215\n",
      "[24]\ttrain-rmse:1.05102\tval-rmse:1.05011\n",
      "[25]\ttrain-rmse:1.04873\tval-rmse:1.04802\n",
      "[26]\ttrain-rmse:1.04649\tval-rmse:1.04600\n",
      "[27]\ttrain-rmse:1.04429\tval-rmse:1.04398\n",
      "[28]\ttrain-rmse:1.04214\tval-rmse:1.04205\n",
      "[29]\ttrain-rmse:1.03999\tval-rmse:1.04009\n",
      "[30]\ttrain-rmse:1.03787\tval-rmse:1.03816\n",
      "[31]\ttrain-rmse:1.03570\tval-rmse:1.03616\n",
      "[32]\ttrain-rmse:1.03362\tval-rmse:1.03429\n",
      "[33]\ttrain-rmse:1.03153\tval-rmse:1.03241\n",
      "[34]\ttrain-rmse:1.02947\tval-rmse:1.03053\n",
      "[35]\ttrain-rmse:1.02745\tval-rmse:1.02868\n",
      "[36]\ttrain-rmse:1.02537\tval-rmse:1.02678\n",
      "[37]\ttrain-rmse:1.02329\tval-rmse:1.02480\n",
      "[38]\ttrain-rmse:1.02124\tval-rmse:1.02294\n",
      "[39]\ttrain-rmse:1.01930\tval-rmse:1.02125\n",
      "[40]\ttrain-rmse:1.01727\tval-rmse:1.01946\n",
      "[41]\ttrain-rmse:1.01531\tval-rmse:1.01769\n",
      "[42]\ttrain-rmse:1.01328\tval-rmse:1.01584\n",
      "[43]\ttrain-rmse:1.01131\tval-rmse:1.01403\n",
      "[44]\ttrain-rmse:1.00931\tval-rmse:1.01216\n",
      "[45]\ttrain-rmse:1.00737\tval-rmse:1.01039\n",
      "[46]\ttrain-rmse:1.00548\tval-rmse:1.00869\n",
      "[47]\ttrain-rmse:1.00349\tval-rmse:1.00691\n",
      "[48]\ttrain-rmse:1.00159\tval-rmse:1.00517\n",
      "[49]\ttrain-rmse:0.99967\tval-rmse:1.00345\n",
      "[50]\ttrain-rmse:0.99774\tval-rmse:1.00165\n",
      "[51]\ttrain-rmse:0.99584\tval-rmse:0.99994\n",
      "[52]\ttrain-rmse:0.99397\tval-rmse:0.99818\n",
      "[53]\ttrain-rmse:0.99209\tval-rmse:0.99649\n",
      "[54]\ttrain-rmse:0.99020\tval-rmse:0.99482\n",
      "[55]\ttrain-rmse:0.98833\tval-rmse:0.99308\n",
      "[56]\ttrain-rmse:0.98649\tval-rmse:0.99140\n",
      "[57]\ttrain-rmse:0.98468\tval-rmse:0.98977\n",
      "[58]\ttrain-rmse:0.98293\tval-rmse:0.98820\n",
      "[59]\ttrain-rmse:0.98113\tval-rmse:0.98655\n",
      "[60]\ttrain-rmse:0.97933\tval-rmse:0.98492\n",
      "[61]\ttrain-rmse:0.97754\tval-rmse:0.98337\n",
      "[62]\ttrain-rmse:0.97566\tval-rmse:0.98167\n",
      "[63]\ttrain-rmse:0.97399\tval-rmse:0.98020\n",
      "[64]\ttrain-rmse:0.97229\tval-rmse:0.97869\n",
      "[65]\ttrain-rmse:0.97051\tval-rmse:0.97715\n",
      "[66]\ttrain-rmse:0.96875\tval-rmse:0.97555\n",
      "[67]\ttrain-rmse:0.96706\tval-rmse:0.97396\n",
      "[68]\ttrain-rmse:0.96543\tval-rmse:0.97249\n",
      "[69]\ttrain-rmse:0.96373\tval-rmse:0.97092\n",
      "[70]\ttrain-rmse:0.96207\tval-rmse:0.96938\n",
      "[71]\ttrain-rmse:0.96044\tval-rmse:0.96796\n",
      "[72]\ttrain-rmse:0.95881\tval-rmse:0.96649\n",
      "[73]\ttrain-rmse:0.95715\tval-rmse:0.96497\n",
      "[74]\ttrain-rmse:0.95554\tval-rmse:0.96354\n",
      "[75]\ttrain-rmse:0.95389\tval-rmse:0.96215\n",
      "[76]\ttrain-rmse:0.95216\tval-rmse:0.96062\n",
      "[77]\ttrain-rmse:0.95056\tval-rmse:0.95913\n",
      "[78]\ttrain-rmse:0.94890\tval-rmse:0.95758\n",
      "[79]\ttrain-rmse:0.94729\tval-rmse:0.95609\n",
      "[80]\ttrain-rmse:0.94567\tval-rmse:0.95468\n",
      "[81]\ttrain-rmse:0.94416\tval-rmse:0.95334\n",
      "[82]\ttrain-rmse:0.94261\tval-rmse:0.95194\n",
      "[83]\ttrain-rmse:0.94103\tval-rmse:0.95055\n",
      "[84]\ttrain-rmse:0.93942\tval-rmse:0.94913\n",
      "[85]\ttrain-rmse:0.93790\tval-rmse:0.94772\n",
      "[86]\ttrain-rmse:0.93637\tval-rmse:0.94639\n",
      "[87]\ttrain-rmse:0.93485\tval-rmse:0.94504\n",
      "[88]\ttrain-rmse:0.93331\tval-rmse:0.94367\n",
      "[89]\ttrain-rmse:0.93183\tval-rmse:0.94238\n",
      "[90]\ttrain-rmse:0.93032\tval-rmse:0.94103\n",
      "[91]\ttrain-rmse:0.92883\tval-rmse:0.93970\n",
      "[92]\ttrain-rmse:0.92737\tval-rmse:0.93836\n",
      "[93]\ttrain-rmse:0.92584\tval-rmse:0.93707\n",
      "[94]\ttrain-rmse:0.92442\tval-rmse:0.93583\n",
      "[95]\ttrain-rmse:0.92303\tval-rmse:0.93458\n",
      "[96]\ttrain-rmse:0.92167\tval-rmse:0.93333\n",
      "[97]\ttrain-rmse:0.92022\tval-rmse:0.93210\n",
      "[98]\ttrain-rmse:0.91876\tval-rmse:0.93081\n",
      "[99]\ttrain-rmse:0.91732\tval-rmse:0.92943\n",
      "[100]\ttrain-rmse:0.91587\tval-rmse:0.92822\n",
      "[101]\ttrain-rmse:0.91452\tval-rmse:0.92695\n",
      "[102]\ttrain-rmse:0.91312\tval-rmse:0.92576\n",
      "[103]\ttrain-rmse:0.91172\tval-rmse:0.92450\n",
      "[104]\ttrain-rmse:0.91039\tval-rmse:0.92334\n",
      "[105]\ttrain-rmse:0.90900\tval-rmse:0.92216\n",
      "[106]\ttrain-rmse:0.90758\tval-rmse:0.92102\n",
      "[107]\ttrain-rmse:0.90620\tval-rmse:0.91982\n",
      "[108]\ttrain-rmse:0.90483\tval-rmse:0.91866\n",
      "[109]\ttrain-rmse:0.90349\tval-rmse:0.91743\n",
      "[110]\ttrain-rmse:0.90210\tval-rmse:0.91619\n",
      "[111]\ttrain-rmse:0.90079\tval-rmse:0.91500\n",
      "[112]\ttrain-rmse:0.89943\tval-rmse:0.91385\n",
      "[113]\ttrain-rmse:0.89816\tval-rmse:0.91276\n",
      "[114]\ttrain-rmse:0.89691\tval-rmse:0.91167\n",
      "[115]\ttrain-rmse:0.89564\tval-rmse:0.91063\n",
      "[116]\ttrain-rmse:0.89438\tval-rmse:0.90948\n",
      "[117]\ttrain-rmse:0.89305\tval-rmse:0.90836\n",
      "[118]\ttrain-rmse:0.89179\tval-rmse:0.90727\n",
      "[119]\ttrain-rmse:0.89047\tval-rmse:0.90606\n",
      "[120]\ttrain-rmse:0.88915\tval-rmse:0.90497\n",
      "[121]\ttrain-rmse:0.88791\tval-rmse:0.90391\n",
      "[122]\ttrain-rmse:0.88666\tval-rmse:0.90275\n",
      "[123]\ttrain-rmse:0.88538\tval-rmse:0.90158\n",
      "[124]\ttrain-rmse:0.88409\tval-rmse:0.90045\n",
      "[125]\ttrain-rmse:0.88287\tval-rmse:0.89940\n",
      "[126]\ttrain-rmse:0.88154\tval-rmse:0.89830\n",
      "[127]\ttrain-rmse:0.88038\tval-rmse:0.89720\n",
      "[128]\ttrain-rmse:0.87913\tval-rmse:0.89609\n",
      "[129]\ttrain-rmse:0.87789\tval-rmse:0.89501\n",
      "[130]\ttrain-rmse:0.87664\tval-rmse:0.89395\n",
      "[131]\ttrain-rmse:0.87543\tval-rmse:0.89289\n",
      "[132]\ttrain-rmse:0.87419\tval-rmse:0.89177\n",
      "[133]\ttrain-rmse:0.87299\tval-rmse:0.89080\n",
      "[134]\ttrain-rmse:0.87177\tval-rmse:0.88981\n",
      "[135]\ttrain-rmse:0.87063\tval-rmse:0.88880\n",
      "[136]\ttrain-rmse:0.86944\tval-rmse:0.88777\n",
      "[137]\ttrain-rmse:0.86829\tval-rmse:0.88667\n",
      "[138]\ttrain-rmse:0.86718\tval-rmse:0.88571\n",
      "[139]\ttrain-rmse:0.86601\tval-rmse:0.88461\n",
      "[140]\ttrain-rmse:0.86483\tval-rmse:0.88355\n",
      "[141]\ttrain-rmse:0.86364\tval-rmse:0.88260\n",
      "[142]\ttrain-rmse:0.86244\tval-rmse:0.88166\n",
      "[143]\ttrain-rmse:0.86135\tval-rmse:0.88068\n",
      "[144]\ttrain-rmse:0.86024\tval-rmse:0.87967\n",
      "[145]\ttrain-rmse:0.85906\tval-rmse:0.87862\n",
      "[146]\ttrain-rmse:0.85793\tval-rmse:0.87754\n",
      "[147]\ttrain-rmse:0.85677\tval-rmse:0.87655\n",
      "[148]\ttrain-rmse:0.85570\tval-rmse:0.87570\n",
      "[149]\ttrain-rmse:0.85458\tval-rmse:0.87474\n",
      "[150]\ttrain-rmse:0.85347\tval-rmse:0.87378\n",
      "[151]\ttrain-rmse:0.85242\tval-rmse:0.87292\n",
      "[152]\ttrain-rmse:0.85134\tval-rmse:0.87200\n",
      "[153]\ttrain-rmse:0.85023\tval-rmse:0.87108\n",
      "[154]\ttrain-rmse:0.84911\tval-rmse:0.87012\n",
      "[155]\ttrain-rmse:0.84802\tval-rmse:0.86925\n",
      "[156]\ttrain-rmse:0.84700\tval-rmse:0.86835\n",
      "[157]\ttrain-rmse:0.84590\tval-rmse:0.86743\n",
      "[158]\ttrain-rmse:0.84482\tval-rmse:0.86651\n",
      "[159]\ttrain-rmse:0.84374\tval-rmse:0.86555\n",
      "[160]\ttrain-rmse:0.84267\tval-rmse:0.86459\n",
      "[161]\ttrain-rmse:0.84166\tval-rmse:0.86370\n",
      "[162]\ttrain-rmse:0.84059\tval-rmse:0.86280\n",
      "[163]\ttrain-rmse:0.83950\tval-rmse:0.86185\n",
      "[164]\ttrain-rmse:0.83843\tval-rmse:0.86096\n",
      "[165]\ttrain-rmse:0.83736\tval-rmse:0.86002\n",
      "[166]\ttrain-rmse:0.83640\tval-rmse:0.85921\n",
      "[167]\ttrain-rmse:0.83533\tval-rmse:0.85831\n",
      "[168]\ttrain-rmse:0.83429\tval-rmse:0.85740\n",
      "[169]\ttrain-rmse:0.83318\tval-rmse:0.85650\n",
      "[170]\ttrain-rmse:0.83215\tval-rmse:0.85553\n",
      "[171]\ttrain-rmse:0.83112\tval-rmse:0.85465\n",
      "[172]\ttrain-rmse:0.83010\tval-rmse:0.85378\n",
      "[173]\ttrain-rmse:0.82908\tval-rmse:0.85298\n",
      "[174]\ttrain-rmse:0.82806\tval-rmse:0.85203\n",
      "[175]\ttrain-rmse:0.82705\tval-rmse:0.85117\n",
      "[176]\ttrain-rmse:0.82604\tval-rmse:0.85034\n",
      "[177]\ttrain-rmse:0.82509\tval-rmse:0.84950\n",
      "[178]\ttrain-rmse:0.82406\tval-rmse:0.84869\n",
      "[179]\ttrain-rmse:0.82307\tval-rmse:0.84776\n",
      "[180]\ttrain-rmse:0.82202\tval-rmse:0.84692\n",
      "[181]\ttrain-rmse:0.82106\tval-rmse:0.84610\n",
      "[182]\ttrain-rmse:0.82008\tval-rmse:0.84532\n",
      "[183]\ttrain-rmse:0.81906\tval-rmse:0.84441\n",
      "[184]\ttrain-rmse:0.81812\tval-rmse:0.84361\n",
      "[185]\ttrain-rmse:0.81716\tval-rmse:0.84278\n",
      "[186]\ttrain-rmse:0.81622\tval-rmse:0.84202\n",
      "[187]\ttrain-rmse:0.81528\tval-rmse:0.84120\n",
      "[188]\ttrain-rmse:0.81436\tval-rmse:0.84043\n",
      "[189]\ttrain-rmse:0.81345\tval-rmse:0.83962\n",
      "[190]\ttrain-rmse:0.81251\tval-rmse:0.83876\n",
      "[191]\ttrain-rmse:0.81163\tval-rmse:0.83807\n",
      "[192]\ttrain-rmse:0.81071\tval-rmse:0.83732\n",
      "[193]\ttrain-rmse:0.80981\tval-rmse:0.83657\n",
      "[194]\ttrain-rmse:0.80887\tval-rmse:0.83574\n",
      "[195]\ttrain-rmse:0.80799\tval-rmse:0.83500\n",
      "[196]\ttrain-rmse:0.80707\tval-rmse:0.83423\n",
      "[197]\ttrain-rmse:0.80613\tval-rmse:0.83340\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[198]\ttrain-rmse:0.80524\tval-rmse:0.83263\n",
      "[199]\ttrain-rmse:0.80434\tval-rmse:0.83187\n",
      "[200]\ttrain-rmse:0.80341\tval-rmse:0.83110\n",
      "[201]\ttrain-rmse:0.80253\tval-rmse:0.83037\n",
      "[202]\ttrain-rmse:0.80169\tval-rmse:0.82971\n",
      "[203]\ttrain-rmse:0.80081\tval-rmse:0.82901\n",
      "[204]\ttrain-rmse:0.79989\tval-rmse:0.82820\n",
      "[205]\ttrain-rmse:0.79902\tval-rmse:0.82740\n",
      "[206]\ttrain-rmse:0.79810\tval-rmse:0.82662\n",
      "[207]\ttrain-rmse:0.79720\tval-rmse:0.82591\n",
      "[208]\ttrain-rmse:0.79630\tval-rmse:0.82514\n",
      "[209]\ttrain-rmse:0.79539\tval-rmse:0.82439\n",
      "[210]\ttrain-rmse:0.79449\tval-rmse:0.82371\n",
      "[211]\ttrain-rmse:0.79358\tval-rmse:0.82297\n",
      "[212]\ttrain-rmse:0.79266\tval-rmse:0.82221\n",
      "[213]\ttrain-rmse:0.79181\tval-rmse:0.82154\n",
      "[214]\ttrain-rmse:0.79094\tval-rmse:0.82078\n",
      "[215]\ttrain-rmse:0.79004\tval-rmse:0.82007\n",
      "[216]\ttrain-rmse:0.78916\tval-rmse:0.81934\n",
      "[217]\ttrain-rmse:0.78831\tval-rmse:0.81866\n",
      "[218]\ttrain-rmse:0.78744\tval-rmse:0.81797\n",
      "[219]\ttrain-rmse:0.78657\tval-rmse:0.81720\n",
      "[220]\ttrain-rmse:0.78569\tval-rmse:0.81650\n",
      "[221]\ttrain-rmse:0.78481\tval-rmse:0.81576\n",
      "[222]\ttrain-rmse:0.78401\tval-rmse:0.81510\n",
      "[223]\ttrain-rmse:0.78317\tval-rmse:0.81442\n",
      "[224]\ttrain-rmse:0.78234\tval-rmse:0.81369\n",
      "[225]\ttrain-rmse:0.78151\tval-rmse:0.81299\n",
      "[226]\ttrain-rmse:0.78065\tval-rmse:0.81228\n",
      "[227]\ttrain-rmse:0.77982\tval-rmse:0.81157\n",
      "[228]\ttrain-rmse:0.77894\tval-rmse:0.81082\n",
      "[229]\ttrain-rmse:0.77807\tval-rmse:0.81016\n",
      "[230]\ttrain-rmse:0.77723\tval-rmse:0.80948\n",
      "[231]\ttrain-rmse:0.77640\tval-rmse:0.80883\n",
      "[232]\ttrain-rmse:0.77556\tval-rmse:0.80811\n",
      "[233]\ttrain-rmse:0.77478\tval-rmse:0.80753\n",
      "[234]\ttrain-rmse:0.77391\tval-rmse:0.80681\n",
      "[235]\ttrain-rmse:0.77306\tval-rmse:0.80612\n",
      "[236]\ttrain-rmse:0.77220\tval-rmse:0.80543\n",
      "[237]\ttrain-rmse:0.77139\tval-rmse:0.80475\n",
      "[238]\ttrain-rmse:0.77065\tval-rmse:0.80412\n",
      "[239]\ttrain-rmse:0.76984\tval-rmse:0.80342\n",
      "[240]\ttrain-rmse:0.76899\tval-rmse:0.80278\n",
      "[241]\ttrain-rmse:0.76826\tval-rmse:0.80216\n",
      "[242]\ttrain-rmse:0.76748\tval-rmse:0.80156\n",
      "[243]\ttrain-rmse:0.76669\tval-rmse:0.80094\n",
      "[244]\ttrain-rmse:0.76589\tval-rmse:0.80029\n",
      "[245]\ttrain-rmse:0.76510\tval-rmse:0.79968\n",
      "[246]\ttrain-rmse:0.76430\tval-rmse:0.79907\n",
      "[247]\ttrain-rmse:0.76350\tval-rmse:0.79839\n",
      "[248]\ttrain-rmse:0.76277\tval-rmse:0.79776\n",
      "[249]\ttrain-rmse:0.76205\tval-rmse:0.79716\n",
      "[250]\ttrain-rmse:0.76125\tval-rmse:0.79652\n",
      "[251]\ttrain-rmse:0.76044\tval-rmse:0.79594\n",
      "[252]\ttrain-rmse:0.75961\tval-rmse:0.79526\n",
      "[253]\ttrain-rmse:0.75883\tval-rmse:0.79455\n",
      "[254]\ttrain-rmse:0.75804\tval-rmse:0.79391\n",
      "[255]\ttrain-rmse:0.75733\tval-rmse:0.79328\n",
      "[256]\ttrain-rmse:0.75655\tval-rmse:0.79261\n",
      "[257]\ttrain-rmse:0.75579\tval-rmse:0.79197\n",
      "[258]\ttrain-rmse:0.75506\tval-rmse:0.79140\n",
      "[259]\ttrain-rmse:0.75429\tval-rmse:0.79079\n",
      "[260]\ttrain-rmse:0.75358\tval-rmse:0.79017\n",
      "[261]\ttrain-rmse:0.75279\tval-rmse:0.78952\n",
      "[262]\ttrain-rmse:0.75203\tval-rmse:0.78888\n",
      "[263]\ttrain-rmse:0.75121\tval-rmse:0.78819\n",
      "[264]\ttrain-rmse:0.75048\tval-rmse:0.78750\n",
      "[265]\ttrain-rmse:0.74974\tval-rmse:0.78687\n",
      "[266]\ttrain-rmse:0.74903\tval-rmse:0.78629\n",
      "[267]\ttrain-rmse:0.74825\tval-rmse:0.78565\n",
      "[268]\ttrain-rmse:0.74748\tval-rmse:0.78506\n",
      "[269]\ttrain-rmse:0.74678\tval-rmse:0.78448\n",
      "[270]\ttrain-rmse:0.74609\tval-rmse:0.78392\n",
      "[271]\ttrain-rmse:0.74541\tval-rmse:0.78334\n",
      "[272]\ttrain-rmse:0.74472\tval-rmse:0.78274\n",
      "[273]\ttrain-rmse:0.74404\tval-rmse:0.78212\n",
      "[274]\ttrain-rmse:0.74327\tval-rmse:0.78156\n",
      "[275]\ttrain-rmse:0.74258\tval-rmse:0.78095\n",
      "[276]\ttrain-rmse:0.74189\tval-rmse:0.78042\n",
      "[277]\ttrain-rmse:0.74118\tval-rmse:0.77985\n",
      "[278]\ttrain-rmse:0.74044\tval-rmse:0.77924\n",
      "[279]\ttrain-rmse:0.73974\tval-rmse:0.77862\n",
      "[280]\ttrain-rmse:0.73906\tval-rmse:0.77810\n",
      "[281]\ttrain-rmse:0.73836\tval-rmse:0.77758\n",
      "[282]\ttrain-rmse:0.73765\tval-rmse:0.77707\n",
      "[283]\ttrain-rmse:0.73695\tval-rmse:0.77647\n",
      "[284]\ttrain-rmse:0.73624\tval-rmse:0.77587\n",
      "[285]\ttrain-rmse:0.73555\tval-rmse:0.77527\n",
      "[286]\ttrain-rmse:0.73486\tval-rmse:0.77468\n",
      "[287]\ttrain-rmse:0.73419\tval-rmse:0.77419\n",
      "[288]\ttrain-rmse:0.73352\tval-rmse:0.77366\n",
      "[289]\ttrain-rmse:0.73280\tval-rmse:0.77304\n",
      "[290]\ttrain-rmse:0.73211\tval-rmse:0.77246\n",
      "[291]\ttrain-rmse:0.73140\tval-rmse:0.77185\n",
      "[292]\ttrain-rmse:0.73073\tval-rmse:0.77128\n",
      "[293]\ttrain-rmse:0.72999\tval-rmse:0.77067\n",
      "[294]\ttrain-rmse:0.72931\tval-rmse:0.77017\n",
      "[295]\ttrain-rmse:0.72864\tval-rmse:0.76965\n",
      "[296]\ttrain-rmse:0.72794\tval-rmse:0.76912\n",
      "[297]\ttrain-rmse:0.72727\tval-rmse:0.76858\n",
      "[298]\ttrain-rmse:0.72660\tval-rmse:0.76802\n",
      "[299]\ttrain-rmse:0.72588\tval-rmse:0.76744\n",
      "[300]\ttrain-rmse:0.72522\tval-rmse:0.76698\n",
      "[301]\ttrain-rmse:0.72461\tval-rmse:0.76649\n",
      "[302]\ttrain-rmse:0.72391\tval-rmse:0.76595\n",
      "[303]\ttrain-rmse:0.72326\tval-rmse:0.76543\n",
      "[304]\ttrain-rmse:0.72259\tval-rmse:0.76487\n",
      "[305]\ttrain-rmse:0.72195\tval-rmse:0.76435\n",
      "[306]\ttrain-rmse:0.72132\tval-rmse:0.76377\n",
      "[307]\ttrain-rmse:0.72066\tval-rmse:0.76324\n",
      "[308]\ttrain-rmse:0.72004\tval-rmse:0.76272\n",
      "[309]\ttrain-rmse:0.71935\tval-rmse:0.76219\n",
      "[310]\ttrain-rmse:0.71868\tval-rmse:0.76165\n",
      "[311]\ttrain-rmse:0.71806\tval-rmse:0.76111\n",
      "[312]\ttrain-rmse:0.71741\tval-rmse:0.76065\n",
      "[313]\ttrain-rmse:0.71674\tval-rmse:0.76006\n",
      "[314]\ttrain-rmse:0.71608\tval-rmse:0.75959\n",
      "[315]\ttrain-rmse:0.71547\tval-rmse:0.75906\n",
      "[316]\ttrain-rmse:0.71478\tval-rmse:0.75852\n",
      "[317]\ttrain-rmse:0.71414\tval-rmse:0.75798\n",
      "[318]\ttrain-rmse:0.71352\tval-rmse:0.75745\n",
      "[319]\ttrain-rmse:0.71287\tval-rmse:0.75697\n",
      "[320]\ttrain-rmse:0.71226\tval-rmse:0.75643\n",
      "[321]\ttrain-rmse:0.71162\tval-rmse:0.75597\n",
      "[322]\ttrain-rmse:0.71096\tval-rmse:0.75548\n",
      "[323]\ttrain-rmse:0.71036\tval-rmse:0.75501\n",
      "[324]\ttrain-rmse:0.70976\tval-rmse:0.75457\n",
      "[325]\ttrain-rmse:0.70910\tval-rmse:0.75402\n",
      "[326]\ttrain-rmse:0.70848\tval-rmse:0.75350\n",
      "[327]\ttrain-rmse:0.70789\tval-rmse:0.75308\n",
      "[328]\ttrain-rmse:0.70721\tval-rmse:0.75264\n",
      "[329]\ttrain-rmse:0.70661\tval-rmse:0.75216\n",
      "[330]\ttrain-rmse:0.70597\tval-rmse:0.75162\n",
      "[331]\ttrain-rmse:0.70536\tval-rmse:0.75114\n",
      "[332]\ttrain-rmse:0.70476\tval-rmse:0.75069\n",
      "[333]\ttrain-rmse:0.70416\tval-rmse:0.75022\n",
      "[334]\ttrain-rmse:0.70352\tval-rmse:0.74974\n",
      "[335]\ttrain-rmse:0.70289\tval-rmse:0.74921\n",
      "[336]\ttrain-rmse:0.70229\tval-rmse:0.74870\n",
      "[337]\ttrain-rmse:0.70173\tval-rmse:0.74823\n",
      "[338]\ttrain-rmse:0.70112\tval-rmse:0.74774\n",
      "[339]\ttrain-rmse:0.70053\tval-rmse:0.74730\n",
      "[340]\ttrain-rmse:0.69992\tval-rmse:0.74682\n",
      "[341]\ttrain-rmse:0.69928\tval-rmse:0.74625\n",
      "[342]\ttrain-rmse:0.69869\tval-rmse:0.74576\n",
      "[343]\ttrain-rmse:0.69812\tval-rmse:0.74532\n",
      "[344]\ttrain-rmse:0.69751\tval-rmse:0.74484\n",
      "[345]\ttrain-rmse:0.69691\tval-rmse:0.74435\n",
      "[346]\ttrain-rmse:0.69631\tval-rmse:0.74392\n",
      "[347]\ttrain-rmse:0.69571\tval-rmse:0.74345\n",
      "[348]\ttrain-rmse:0.69513\tval-rmse:0.74295\n",
      "[349]\ttrain-rmse:0.69460\tval-rmse:0.74253\n",
      "[350]\ttrain-rmse:0.69400\tval-rmse:0.74205\n",
      "[351]\ttrain-rmse:0.69339\tval-rmse:0.74158\n",
      "[352]\ttrain-rmse:0.69281\tval-rmse:0.74109\n",
      "[353]\ttrain-rmse:0.69223\tval-rmse:0.74060\n",
      "[354]\ttrain-rmse:0.69158\tval-rmse:0.74009\n",
      "[355]\ttrain-rmse:0.69102\tval-rmse:0.73968\n",
      "[356]\ttrain-rmse:0.69046\tval-rmse:0.73923\n",
      "[357]\ttrain-rmse:0.68992\tval-rmse:0.73881\n",
      "[358]\ttrain-rmse:0.68933\tval-rmse:0.73831\n",
      "[359]\ttrain-rmse:0.68873\tval-rmse:0.73781\n",
      "[360]\ttrain-rmse:0.68812\tval-rmse:0.73739\n",
      "[361]\ttrain-rmse:0.68752\tval-rmse:0.73687\n",
      "[362]\ttrain-rmse:0.68697\tval-rmse:0.73640\n",
      "[363]\ttrain-rmse:0.68644\tval-rmse:0.73595\n",
      "[364]\ttrain-rmse:0.68587\tval-rmse:0.73549\n",
      "[365]\ttrain-rmse:0.68528\tval-rmse:0.73510\n",
      "[366]\ttrain-rmse:0.68469\tval-rmse:0.73460\n",
      "[367]\ttrain-rmse:0.68413\tval-rmse:0.73410\n",
      "[368]\ttrain-rmse:0.68362\tval-rmse:0.73371\n",
      "[369]\ttrain-rmse:0.68303\tval-rmse:0.73321\n",
      "[370]\ttrain-rmse:0.68246\tval-rmse:0.73276\n",
      "[371]\ttrain-rmse:0.68188\tval-rmse:0.73234\n",
      "[372]\ttrain-rmse:0.68133\tval-rmse:0.73191\n",
      "[373]\ttrain-rmse:0.68082\tval-rmse:0.73148\n",
      "[374]\ttrain-rmse:0.68026\tval-rmse:0.73110\n",
      "[375]\ttrain-rmse:0.67970\tval-rmse:0.73072\n",
      "[376]\ttrain-rmse:0.67908\tval-rmse:0.73022\n",
      "[377]\ttrain-rmse:0.67856\tval-rmse:0.72977\n",
      "[378]\ttrain-rmse:0.67799\tval-rmse:0.72935\n",
      "[379]\ttrain-rmse:0.67742\tval-rmse:0.72894\n",
      "[380]\ttrain-rmse:0.67687\tval-rmse:0.72850\n",
      "[381]\ttrain-rmse:0.67631\tval-rmse:0.72807\n",
      "[382]\ttrain-rmse:0.67577\tval-rmse:0.72763\n",
      "[383]\ttrain-rmse:0.67523\tval-rmse:0.72717\n",
      "[384]\ttrain-rmse:0.67475\tval-rmse:0.72677\n",
      "[385]\ttrain-rmse:0.67423\tval-rmse:0.72637\n",
      "[386]\ttrain-rmse:0.67368\tval-rmse:0.72598\n",
      "[387]\ttrain-rmse:0.67312\tval-rmse:0.72557\n",
      "[388]\ttrain-rmse:0.67265\tval-rmse:0.72520\n",
      "[389]\ttrain-rmse:0.67212\tval-rmse:0.72478\n",
      "[390]\ttrain-rmse:0.67160\tval-rmse:0.72435\n",
      "[391]\ttrain-rmse:0.67107\tval-rmse:0.72397\n",
      "[392]\ttrain-rmse:0.67054\tval-rmse:0.72352\n",
      "[393]\ttrain-rmse:0.67000\tval-rmse:0.72310\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[394]\ttrain-rmse:0.66944\tval-rmse:0.72270\n",
      "[395]\ttrain-rmse:0.66893\tval-rmse:0.72232\n",
      "[396]\ttrain-rmse:0.66842\tval-rmse:0.72194\n",
      "[397]\ttrain-rmse:0.66787\tval-rmse:0.72160\n",
      "[398]\ttrain-rmse:0.66731\tval-rmse:0.72117\n",
      "[399]\ttrain-rmse:0.66681\tval-rmse:0.72068\n",
      "[400]\ttrain-rmse:0.66629\tval-rmse:0.72026\n",
      "[401]\ttrain-rmse:0.66578\tval-rmse:0.71988\n",
      "[402]\ttrain-rmse:0.66529\tval-rmse:0.71948\n",
      "[403]\ttrain-rmse:0.66479\tval-rmse:0.71908\n",
      "[404]\ttrain-rmse:0.66428\tval-rmse:0.71869\n",
      "[405]\ttrain-rmse:0.66381\tval-rmse:0.71831\n",
      "[406]\ttrain-rmse:0.66330\tval-rmse:0.71789\n",
      "[407]\ttrain-rmse:0.66278\tval-rmse:0.71750\n",
      "[408]\ttrain-rmse:0.66230\tval-rmse:0.71715\n",
      "[409]\ttrain-rmse:0.66180\tval-rmse:0.71676\n",
      "[410]\ttrain-rmse:0.66126\tval-rmse:0.71635\n",
      "[411]\ttrain-rmse:0.66074\tval-rmse:0.71596\n",
      "[412]\ttrain-rmse:0.66029\tval-rmse:0.71567\n",
      "[413]\ttrain-rmse:0.65978\tval-rmse:0.71532\n",
      "[414]\ttrain-rmse:0.65927\tval-rmse:0.71490\n",
      "[415]\ttrain-rmse:0.65876\tval-rmse:0.71450\n",
      "[416]\ttrain-rmse:0.65825\tval-rmse:0.71415\n",
      "[417]\ttrain-rmse:0.65780\tval-rmse:0.71382\n",
      "[418]\ttrain-rmse:0.65730\tval-rmse:0.71341\n",
      "[419]\ttrain-rmse:0.65680\tval-rmse:0.71302\n",
      "[420]\ttrain-rmse:0.65630\tval-rmse:0.71267\n",
      "[421]\ttrain-rmse:0.65581\tval-rmse:0.71230\n",
      "[422]\ttrain-rmse:0.65533\tval-rmse:0.71196\n",
      "[423]\ttrain-rmse:0.65483\tval-rmse:0.71158\n",
      "[424]\ttrain-rmse:0.65438\tval-rmse:0.71127\n",
      "[425]\ttrain-rmse:0.65392\tval-rmse:0.71088\n",
      "[426]\ttrain-rmse:0.65342\tval-rmse:0.71052\n",
      "[427]\ttrain-rmse:0.65295\tval-rmse:0.71014\n",
      "[428]\ttrain-rmse:0.65251\tval-rmse:0.70984\n",
      "[429]\ttrain-rmse:0.65206\tval-rmse:0.70955\n",
      "[430]\ttrain-rmse:0.65159\tval-rmse:0.70922\n",
      "[431]\ttrain-rmse:0.65111\tval-rmse:0.70883\n",
      "[432]\ttrain-rmse:0.65062\tval-rmse:0.70844\n",
      "[433]\ttrain-rmse:0.65017\tval-rmse:0.70810\n",
      "[434]\ttrain-rmse:0.64969\tval-rmse:0.70771\n",
      "[435]\ttrain-rmse:0.64922\tval-rmse:0.70731\n",
      "[436]\ttrain-rmse:0.64873\tval-rmse:0.70700\n",
      "[437]\ttrain-rmse:0.64824\tval-rmse:0.70668\n",
      "[438]\ttrain-rmse:0.64777\tval-rmse:0.70635\n",
      "[439]\ttrain-rmse:0.64724\tval-rmse:0.70601\n",
      "[440]\ttrain-rmse:0.64678\tval-rmse:0.70562\n",
      "[441]\ttrain-rmse:0.64631\tval-rmse:0.70523\n",
      "[442]\ttrain-rmse:0.64583\tval-rmse:0.70490\n",
      "[443]\ttrain-rmse:0.64535\tval-rmse:0.70455\n",
      "[444]\ttrain-rmse:0.64487\tval-rmse:0.70419\n",
      "[445]\ttrain-rmse:0.64441\tval-rmse:0.70384\n",
      "[446]\ttrain-rmse:0.64396\tval-rmse:0.70349\n",
      "[447]\ttrain-rmse:0.64350\tval-rmse:0.70313\n",
      "[448]\ttrain-rmse:0.64300\tval-rmse:0.70278\n",
      "[449]\ttrain-rmse:0.64254\tval-rmse:0.70250\n",
      "[450]\ttrain-rmse:0.64208\tval-rmse:0.70216\n",
      "[451]\ttrain-rmse:0.64157\tval-rmse:0.70178\n",
      "[452]\ttrain-rmse:0.64109\tval-rmse:0.70144\n",
      "[453]\ttrain-rmse:0.64065\tval-rmse:0.70109\n",
      "[454]\ttrain-rmse:0.64020\tval-rmse:0.70072\n",
      "[455]\ttrain-rmse:0.63973\tval-rmse:0.70036\n",
      "[456]\ttrain-rmse:0.63926\tval-rmse:0.69998\n",
      "[457]\ttrain-rmse:0.63884\tval-rmse:0.69967\n",
      "[458]\ttrain-rmse:0.63840\tval-rmse:0.69938\n",
      "[459]\ttrain-rmse:0.63792\tval-rmse:0.69903\n",
      "[460]\ttrain-rmse:0.63743\tval-rmse:0.69866\n",
      "[461]\ttrain-rmse:0.63695\tval-rmse:0.69831\n",
      "[462]\ttrain-rmse:0.63655\tval-rmse:0.69796\n",
      "[463]\ttrain-rmse:0.63613\tval-rmse:0.69760\n",
      "[464]\ttrain-rmse:0.63567\tval-rmse:0.69726\n",
      "[465]\ttrain-rmse:0.63523\tval-rmse:0.69695\n",
      "[466]\ttrain-rmse:0.63475\tval-rmse:0.69664\n",
      "[467]\ttrain-rmse:0.63430\tval-rmse:0.69625\n",
      "[468]\ttrain-rmse:0.63389\tval-rmse:0.69597\n",
      "[469]\ttrain-rmse:0.63342\tval-rmse:0.69563\n",
      "[470]\ttrain-rmse:0.63299\tval-rmse:0.69538\n",
      "[471]\ttrain-rmse:0.63254\tval-rmse:0.69505\n",
      "[472]\ttrain-rmse:0.63207\tval-rmse:0.69470\n",
      "[473]\ttrain-rmse:0.63164\tval-rmse:0.69443\n",
      "[474]\ttrain-rmse:0.63121\tval-rmse:0.69414\n",
      "[475]\ttrain-rmse:0.63082\tval-rmse:0.69383\n",
      "[476]\ttrain-rmse:0.63035\tval-rmse:0.69350\n",
      "[477]\ttrain-rmse:0.62995\tval-rmse:0.69321\n",
      "[478]\ttrain-rmse:0.62950\tval-rmse:0.69284\n",
      "[479]\ttrain-rmse:0.62902\tval-rmse:0.69252\n",
      "[480]\ttrain-rmse:0.62865\tval-rmse:0.69225\n",
      "[481]\ttrain-rmse:0.62821\tval-rmse:0.69197\n",
      "[482]\ttrain-rmse:0.62776\tval-rmse:0.69165\n",
      "[483]\ttrain-rmse:0.62734\tval-rmse:0.69125\n",
      "[484]\ttrain-rmse:0.62691\tval-rmse:0.69097\n",
      "[485]\ttrain-rmse:0.62651\tval-rmse:0.69066\n",
      "[486]\ttrain-rmse:0.62608\tval-rmse:0.69035\n",
      "[487]\ttrain-rmse:0.62563\tval-rmse:0.69003\n",
      "[488]\ttrain-rmse:0.62519\tval-rmse:0.68971\n",
      "[489]\ttrain-rmse:0.62474\tval-rmse:0.68936\n",
      "[490]\ttrain-rmse:0.62429\tval-rmse:0.68902\n",
      "[491]\ttrain-rmse:0.62389\tval-rmse:0.68866\n",
      "[492]\ttrain-rmse:0.62348\tval-rmse:0.68834\n",
      "[493]\ttrain-rmse:0.62301\tval-rmse:0.68802\n",
      "[494]\ttrain-rmse:0.62263\tval-rmse:0.68772\n",
      "[495]\ttrain-rmse:0.62219\tval-rmse:0.68738\n",
      "[496]\ttrain-rmse:0.62175\tval-rmse:0.68707\n",
      "[497]\ttrain-rmse:0.62135\tval-rmse:0.68680\n",
      "[498]\ttrain-rmse:0.62096\tval-rmse:0.68650\n",
      "[499]\ttrain-rmse:0.62056\tval-rmse:0.68624\n",
      "best best_ntree_limit 500\n",
      "error=0.826667\n",
      "xgboost success! \n",
      " cost time: 3.5742645263671875 (s)......\n"
     ]
    }
   ],
   "source": [
    "#训练模型并保存\n",
    "# early_stopping_rounds 当设置的迭代次数较大时，early_stopping_rounds 可在一定的迭代次数内准确率没有提升就停止训练\n",
    "model = xgb.train(\n",
    "    plst,\n",
    "    xgb_train,\n",
    "    num_rounds,\n",
    "    watchlist,\n",
    "    early_stopping_rounds=100,\n",
    ")\n",
    "#model.save_model('./model/xgb.model') # 用于存储训练出的模型\n",
    "print(\"best best_ntree_limit\", model.best_ntree_limit)\n",
    "y_pred = model.predict(xgb_test, ntree_limit=model.best_ntree_limit)\n",
    "print('error=%f' %\n",
    "      (sum(1\n",
    "           for i in range(len(y_pred)) if int(y_pred[i] > 0.5) != y_test[i]) /\n",
    "       float(len(y_pred))))\n",
    "# 输出运行时长\n",
    "cost_time = time.time() - start_time\n",
    "print(\"xgboost success!\", '\\n', \"cost time:\", cost_time, \"(s)......\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caee89f1",
   "metadata": {},
   "source": [
    "### 2.使用scikit-learn接口\n",
    "会改变的函数名是：\n",
    "\n",
    "eta -> learning_rate\n",
    "\n",
    "lambda -> reg_lambda\n",
    "\n",
    "alpha -> reg_alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dee9b457",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[20:16:02] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy : 0.936\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "clf = XGBClassifier(\n",
    "    #     silent=0,  #设置成1则没有运行信息输出，最好是设置为0.是否在运行升级时打印消息。\n",
    "    #nthread=4,# cpu 线程数 默认最大\n",
    "    learning_rate=0.3,  # 如同学习率\n",
    "    min_child_weight=1,\n",
    "    # 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言\n",
    "    #，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。\n",
    "    #这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。\n",
    "    max_depth=6,  # 构建树的深度，越大越容易过拟合\n",
    "    gamma=0,  # 树的叶子节点上作进一步分区所需的最小损失减少,越大越保守，一般0.1、0.2这样子。\n",
    "    subsample=1,  # 随机采样训练样本 训练实例的子采样比\n",
    "    max_delta_step=0,  #最大增量步长，我们允许每个树的权重估计。\n",
    "    colsample_bytree=1,  # 生成树时进行的列采样 \n",
    "    reg_lambda=1,  # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。\n",
    "    #reg_alpha=0, # L1 正则项参数\n",
    "    #scale_pos_weight=1, #如果取值大于0的话，在类别样本不平衡的情况下有助于快速收敛。平衡正负权重\n",
    "    #objective= 'multi:softmax', #多分类的问题 指定学习任务和相应的学习目标\n",
    "    #num_class=10, # 类别数，多分类与 multisoftmax 并用\n",
    "    n_estimators=100,  #树的个数\n",
    "    seed=1000  #随机种子\n",
    "    #eval_metric= 'auc'\n",
    ")\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(\"Accuracy : %.4g\" % metrics.accuracy_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6196fe2",
   "metadata": {},
   "source": [
    "## LIghtGBM的使用\n",
    "### 1.原生接口"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "04fa87f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training...\n",
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.000448 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 2550\n",
      "[LightGBM] [Info] Number of data points in the train set: 9000, number of used features: 10\n",
      "[LightGBM] [Info] Start training from score 0.012000\n",
      "[1]\tvalid_0's auc: 0.814399\tvalid_0's l2: 0.965563\n",
      "Training until validation scores don't improve for 5 rounds\n",
      "[2]\tvalid_0's auc: 0.84729\tvalid_0's l2: 0.934647\n",
      "[3]\tvalid_0's auc: 0.872805\tvalid_0's l2: 0.905265\n",
      "[4]\tvalid_0's auc: 0.884117\tvalid_0's l2: 0.877875\n",
      "[5]\tvalid_0's auc: 0.895115\tvalid_0's l2: 0.852189\n",
      "[6]\tvalid_0's auc: 0.905545\tvalid_0's l2: 0.826298\n",
      "[7]\tvalid_0's auc: 0.909113\tvalid_0's l2: 0.803776\n",
      "[8]\tvalid_0's auc: 0.913303\tvalid_0's l2: 0.781627\n",
      "[9]\tvalid_0's auc: 0.917894\tvalid_0's l2: 0.760624\n",
      "[10]\tvalid_0's auc: 0.919443\tvalid_0's l2: 0.742882\n",
      "[11]\tvalid_0's auc: 0.921543\tvalid_0's l2: 0.723811\n",
      "[12]\tvalid_0's auc: 0.923021\tvalid_0's l2: 0.707255\n",
      "[13]\tvalid_0's auc: 0.9257\tvalid_0's l2: 0.69078\n",
      "[14]\tvalid_0's auc: 0.928892\tvalid_0's l2: 0.675987\n",
      "[15]\tvalid_0's auc: 0.930132\tvalid_0's l2: 0.661313\n",
      "[16]\tvalid_0's auc: 0.931587\tvalid_0's l2: 0.646023\n",
      "[17]\tvalid_0's auc: 0.932941\tvalid_0's l2: 0.634004\n",
      "[18]\tvalid_0's auc: 0.934165\tvalid_0's l2: 0.622429\n",
      "[19]\tvalid_0's auc: 0.935885\tvalid_0's l2: 0.610132\n",
      "[20]\tvalid_0's auc: 0.936883\tvalid_0's l2: 0.599122\n",
      "[21]\tvalid_0's auc: 0.93814\tvalid_0's l2: 0.589571\n",
      "[22]\tvalid_0's auc: 0.940452\tvalid_0's l2: 0.580309\n",
      "[23]\tvalid_0's auc: 0.941039\tvalid_0's l2: 0.571361\n",
      "[24]\tvalid_0's auc: 0.943049\tvalid_0's l2: 0.562062\n",
      "[25]\tvalid_0's auc: 0.9446\tvalid_0's l2: 0.551967\n",
      "[26]\tvalid_0's auc: 0.946498\tvalid_0's l2: 0.543442\n",
      "[27]\tvalid_0's auc: 0.94763\tvalid_0's l2: 0.535659\n",
      "[28]\tvalid_0's auc: 0.94871\tvalid_0's l2: 0.527913\n",
      "[29]\tvalid_0's auc: 0.949753\tvalid_0's l2: 0.521228\n",
      "[30]\tvalid_0's auc: 0.950816\tvalid_0's l2: 0.513909\n",
      "[31]\tvalid_0's auc: 0.95184\tvalid_0's l2: 0.507784\n",
      "[32]\tvalid_0's auc: 0.953109\tvalid_0's l2: 0.501336\n",
      "[33]\tvalid_0's auc: 0.954351\tvalid_0's l2: 0.494439\n",
      "[34]\tvalid_0's auc: 0.955716\tvalid_0's l2: 0.488722\n",
      "[35]\tvalid_0's auc: 0.956098\tvalid_0's l2: 0.483373\n",
      "[36]\tvalid_0's auc: 0.956495\tvalid_0's l2: 0.477602\n",
      "[37]\tvalid_0's auc: 0.956717\tvalid_0's l2: 0.473033\n",
      "[38]\tvalid_0's auc: 0.957213\tvalid_0's l2: 0.468013\n",
      "[39]\tvalid_0's auc: 0.957812\tvalid_0's l2: 0.463634\n",
      "[40]\tvalid_0's auc: 0.957862\tvalid_0's l2: 0.459433\n",
      "[41]\tvalid_0's auc: 0.958249\tvalid_0's l2: 0.455687\n",
      "[42]\tvalid_0's auc: 0.958799\tvalid_0's l2: 0.450696\n",
      "[43]\tvalid_0's auc: 0.959311\tvalid_0's l2: 0.446838\n",
      "[44]\tvalid_0's auc: 0.959835\tvalid_0's l2: 0.44233\n",
      "[45]\tvalid_0's auc: 0.960234\tvalid_0's l2: 0.438117\n",
      "[46]\tvalid_0's auc: 0.960826\tvalid_0's l2: 0.43469\n",
      "[47]\tvalid_0's auc: 0.961647\tvalid_0's l2: 0.430488\n",
      "[48]\tvalid_0's auc: 0.962359\tvalid_0's l2: 0.427449\n",
      "[49]\tvalid_0's auc: 0.962506\tvalid_0's l2: 0.424433\n",
      "[50]\tvalid_0's auc: 0.962897\tvalid_0's l2: 0.420571\n",
      "[51]\tvalid_0's auc: 0.963657\tvalid_0's l2: 0.417288\n",
      "[52]\tvalid_0's auc: 0.964224\tvalid_0's l2: 0.414743\n",
      "[53]\tvalid_0's auc: 0.964903\tvalid_0's l2: 0.412255\n",
      "[54]\tvalid_0's auc: 0.965508\tvalid_0's l2: 0.40907\n",
      "[55]\tvalid_0's auc: 0.966194\tvalid_0's l2: 0.406477\n",
      "[56]\tvalid_0's auc: 0.966759\tvalid_0's l2: 0.403771\n",
      "[57]\tvalid_0's auc: 0.966901\tvalid_0's l2: 0.400885\n",
      "[58]\tvalid_0's auc: 0.967291\tvalid_0's l2: 0.398386\n",
      "[59]\tvalid_0's auc: 0.967779\tvalid_0's l2: 0.395949\n",
      "[60]\tvalid_0's auc: 0.968119\tvalid_0's l2: 0.393905\n",
      "[61]\tvalid_0's auc: 0.968517\tvalid_0's l2: 0.391743\n",
      "[62]\tvalid_0's auc: 0.968891\tvalid_0's l2: 0.389717\n",
      "[63]\tvalid_0's auc: 0.969304\tvalid_0's l2: 0.387769\n",
      "[64]\tvalid_0's auc: 0.969598\tvalid_0's l2: 0.385498\n",
      "[65]\tvalid_0's auc: 0.969953\tvalid_0's l2: 0.383139\n",
      "[66]\tvalid_0's auc: 0.970443\tvalid_0's l2: 0.38094\n",
      "[67]\tvalid_0's auc: 0.970888\tvalid_0's l2: 0.378793\n",
      "[68]\tvalid_0's auc: 0.971189\tvalid_0's l2: 0.376754\n",
      "[69]\tvalid_0's auc: 0.971377\tvalid_0's l2: 0.37495\n",
      "[70]\tvalid_0's auc: 0.971692\tvalid_0's l2: 0.37324\n",
      "[71]\tvalid_0's auc: 0.971954\tvalid_0's l2: 0.371629\n",
      "[72]\tvalid_0's auc: 0.972278\tvalid_0's l2: 0.370046\n",
      "[73]\tvalid_0's auc: 0.972622\tvalid_0's l2: 0.368577\n",
      "[74]\tvalid_0's auc: 0.972986\tvalid_0's l2: 0.366746\n",
      "[75]\tvalid_0's auc: 0.973308\tvalid_0's l2: 0.365326\n",
      "[76]\tvalid_0's auc: 0.973449\tvalid_0's l2: 0.364078\n",
      "[77]\tvalid_0's auc: 0.973681\tvalid_0's l2: 0.362431\n",
      "[78]\tvalid_0's auc: 0.973941\tvalid_0's l2: 0.361071\n",
      "[79]\tvalid_0's auc: 0.97428\tvalid_0's l2: 0.359825\n",
      "[80]\tvalid_0's auc: 0.974554\tvalid_0's l2: 0.358506\n",
      "[81]\tvalid_0's auc: 0.974731\tvalid_0's l2: 0.357538\n",
      "[82]\tvalid_0's auc: 0.975094\tvalid_0's l2: 0.355998\n",
      "[83]\tvalid_0's auc: 0.97531\tvalid_0's l2: 0.354819\n",
      "[84]\tvalid_0's auc: 0.975363\tvalid_0's l2: 0.353645\n",
      "[85]\tvalid_0's auc: 0.9756\tvalid_0's l2: 0.352575\n",
      "[86]\tvalid_0's auc: 0.975688\tvalid_0's l2: 0.351995\n",
      "[87]\tvalid_0's auc: 0.975909\tvalid_0's l2: 0.350867\n",
      "[88]\tvalid_0's auc: 0.97603\tvalid_0's l2: 0.350146\n",
      "[89]\tvalid_0's auc: 0.976171\tvalid_0's l2: 0.34933\n",
      "[90]\tvalid_0's auc: 0.976264\tvalid_0's l2: 0.348303\n",
      "[91]\tvalid_0's auc: 0.976501\tvalid_0's l2: 0.347415\n",
      "[92]\tvalid_0's auc: 0.976681\tvalid_0's l2: 0.346621\n",
      "[93]\tvalid_0's auc: 0.976794\tvalid_0's l2: 0.345989\n",
      "[94]\tvalid_0's auc: 0.976892\tvalid_0's l2: 0.345124\n",
      "[95]\tvalid_0's auc: 0.977077\tvalid_0's l2: 0.344425\n",
      "[96]\tvalid_0's auc: 0.9771\tvalid_0's l2: 0.343969\n",
      "[97]\tvalid_0's auc: 0.977176\tvalid_0's l2: 0.343221\n",
      "[98]\tvalid_0's auc: 0.977239\tvalid_0's l2: 0.342578\n",
      "[99]\tvalid_0's auc: 0.977433\tvalid_0's l2: 0.341817\n",
      "[100]\tvalid_0's auc: 0.977516\tvalid_0's l2: 0.341229\n",
      "[101]\tvalid_0's auc: 0.977559\tvalid_0's l2: 0.340357\n",
      "[102]\tvalid_0's auc: 0.977707\tvalid_0's l2: 0.339484\n",
      "[103]\tvalid_0's auc: 0.977742\tvalid_0's l2: 0.339004\n",
      "[104]\tvalid_0's auc: 0.977806\tvalid_0's l2: 0.338581\n",
      "[105]\tvalid_0's auc: 0.977983\tvalid_0's l2: 0.338095\n",
      "[106]\tvalid_0's auc: 0.978113\tvalid_0's l2: 0.337505\n",
      "[107]\tvalid_0's auc: 0.978251\tvalid_0's l2: 0.336939\n",
      "[108]\tvalid_0's auc: 0.978479\tvalid_0's l2: 0.336443\n",
      "[109]\tvalid_0's auc: 0.978611\tvalid_0's l2: 0.336062\n",
      "[110]\tvalid_0's auc: 0.978694\tvalid_0's l2: 0.335636\n",
      "[111]\tvalid_0's auc: 0.97885\tvalid_0's l2: 0.335083\n",
      "[112]\tvalid_0's auc: 0.979037\tvalid_0's l2: 0.334435\n",
      "[113]\tvalid_0's auc: 0.979209\tvalid_0's l2: 0.333876\n",
      "[114]\tvalid_0's auc: 0.97939\tvalid_0's l2: 0.333341\n",
      "[115]\tvalid_0's auc: 0.979513\tvalid_0's l2: 0.332968\n",
      "[116]\tvalid_0's auc: 0.979615\tvalid_0's l2: 0.332583\n",
      "[117]\tvalid_0's auc: 0.979741\tvalid_0's l2: 0.332138\n",
      "[118]\tvalid_0's auc: 0.979883\tvalid_0's l2: 0.331546\n",
      "[119]\tvalid_0's auc: 0.979971\tvalid_0's l2: 0.331399\n",
      "[120]\tvalid_0's auc: 0.980002\tvalid_0's l2: 0.331036\n",
      "[121]\tvalid_0's auc: 0.980098\tvalid_0's l2: 0.330674\n",
      "[122]\tvalid_0's auc: 0.980204\tvalid_0's l2: 0.330228\n",
      "[123]\tvalid_0's auc: 0.980204\tvalid_0's l2: 0.330131\n",
      "[124]\tvalid_0's auc: 0.980271\tvalid_0's l2: 0.329895\n",
      "[125]\tvalid_0's auc: 0.980441\tvalid_0's l2: 0.329194\n",
      "[126]\tvalid_0's auc: 0.980456\tvalid_0's l2: 0.328811\n",
      "[127]\tvalid_0's auc: 0.980472\tvalid_0's l2: 0.328493\n",
      "[128]\tvalid_0's auc: 0.980519\tvalid_0's l2: 0.328459\n",
      "[129]\tvalid_0's auc: 0.980578\tvalid_0's l2: 0.32832\n",
      "[130]\tvalid_0's auc: 0.980635\tvalid_0's l2: 0.328198\n",
      "[131]\tvalid_0's auc: 0.980771\tvalid_0's l2: 0.327791\n",
      "[132]\tvalid_0's auc: 0.980872\tvalid_0's l2: 0.327462\n",
      "[133]\tvalid_0's auc: 0.980884\tvalid_0's l2: 0.327269\n",
      "[134]\tvalid_0's auc: 0.980951\tvalid_0's l2: 0.327037\n",
      "[135]\tvalid_0's auc: 0.980989\tvalid_0's l2: 0.326838\n",
      "[136]\tvalid_0's auc: 0.981031\tvalid_0's l2: 0.32665\n",
      "[137]\tvalid_0's auc: 0.981025\tvalid_0's l2: 0.326543\n",
      "[138]\tvalid_0's auc: 0.981099\tvalid_0's l2: 0.326342\n",
      "[139]\tvalid_0's auc: 0.981079\tvalid_0's l2: 0.326256\n",
      "[140]\tvalid_0's auc: 0.981083\tvalid_0's l2: 0.326143\n",
      "[141]\tvalid_0's auc: 0.981149\tvalid_0's l2: 0.32578\n",
      "[142]\tvalid_0's auc: 0.981222\tvalid_0's l2: 0.325428\n",
      "[143]\tvalid_0's auc: 0.98131\tvalid_0's l2: 0.325142\n",
      "[144]\tvalid_0's auc: 0.981348\tvalid_0's l2: 0.324963\n",
      "[145]\tvalid_0's auc: 0.981395\tvalid_0's l2: 0.324856\n",
      "[146]\tvalid_0's auc: 0.981461\tvalid_0's l2: 0.324682\n",
      "[147]\tvalid_0's auc: 0.981544\tvalid_0's l2: 0.324538\n",
      "[148]\tvalid_0's auc: 0.981605\tvalid_0's l2: 0.324309\n",
      "[149]\tvalid_0's auc: 0.981641\tvalid_0's l2: 0.324249\n",
      "[150]\tvalid_0's auc: 0.981707\tvalid_0's l2: 0.324083\n",
      "[151]\tvalid_0's auc: 0.981747\tvalid_0's l2: 0.323942\n",
      "[152]\tvalid_0's auc: 0.981823\tvalid_0's l2: 0.323728\n",
      "[153]\tvalid_0's auc: 0.981888\tvalid_0's l2: 0.323549\n",
      "[154]\tvalid_0's auc: 0.981936\tvalid_0's l2: 0.323444\n",
      "[155]\tvalid_0's auc: 0.982036\tvalid_0's l2: 0.323267\n",
      "[156]\tvalid_0's auc: 0.982064\tvalid_0's l2: 0.323081\n",
      "[157]\tvalid_0's auc: 0.982064\tvalid_0's l2: 0.323087\n",
      "[158]\tvalid_0's auc: 0.982105\tvalid_0's l2: 0.322942\n",
      "[159]\tvalid_0's auc: 0.982106\tvalid_0's l2: 0.322876\n",
      "[160]\tvalid_0's auc: 0.982114\tvalid_0's l2: 0.322758\n",
      "[161]\tvalid_0's auc: 0.982147\tvalid_0's l2: 0.322571\n",
      "[162]\tvalid_0's auc: 0.982193\tvalid_0's l2: 0.322484\n",
      "[163]\tvalid_0's auc: 0.982217\tvalid_0's l2: 0.322336\n",
      "[164]\tvalid_0's auc: 0.982271\tvalid_0's l2: 0.322134\n",
      "[165]\tvalid_0's auc: 0.982268\tvalid_0's l2: 0.322089\n",
      "[166]\tvalid_0's auc: 0.982285\tvalid_0's l2: 0.322113\n",
      "[167]\tvalid_0's auc: 0.982278\tvalid_0's l2: 0.322163\n",
      "[168]\tvalid_0's auc: 0.982341\tvalid_0's l2: 0.322001\n",
      "[169]\tvalid_0's auc: 0.982368\tvalid_0's l2: 0.322046\n",
      "[170]\tvalid_0's auc: 0.982379\tvalid_0's l2: 0.321915\n",
      "[171]\tvalid_0's auc: 0.982344\tvalid_0's l2: 0.32184\n",
      "[172]\tvalid_0's auc: 0.982431\tvalid_0's l2: 0.32163\n",
      "[173]\tvalid_0's auc: 0.982449\tvalid_0's l2: 0.321532\n",
      "[174]\tvalid_0's auc: 0.982469\tvalid_0's l2: 0.321488\n",
      "[175]\tvalid_0's auc: 0.982556\tvalid_0's l2: 0.321291\n",
      "[176]\tvalid_0's auc: 0.982616\tvalid_0's l2: 0.320934\n",
      "[177]\tvalid_0's auc: 0.982631\tvalid_0's l2: 0.320862\n",
      "[178]\tvalid_0's auc: 0.982641\tvalid_0's l2: 0.32074\n",
      "[179]\tvalid_0's auc: 0.982714\tvalid_0's l2: 0.320641\n",
      "[180]\tvalid_0's auc: 0.982727\tvalid_0's l2: 0.320571\n",
      "[181]\tvalid_0's auc: 0.982733\tvalid_0's l2: 0.320354\n",
      "[182]\tvalid_0's auc: 0.982752\tvalid_0's l2: 0.32015\n",
      "[183]\tvalid_0's auc: 0.982776\tvalid_0's l2: 0.320041\n",
      "[184]\tvalid_0's auc: 0.982755\tvalid_0's l2: 0.320013\n",
      "[185]\tvalid_0's auc: 0.982758\tvalid_0's l2: 0.319983\n",
      "[186]\tvalid_0's auc: 0.98273\tvalid_0's l2: 0.320012\n",
      "[187]\tvalid_0's auc: 0.98274\tvalid_0's l2: 0.319916\n",
      "[188]\tvalid_0's auc: 0.982794\tvalid_0's l2: 0.319746\n",
      "[189]\tvalid_0's auc: 0.982785\tvalid_0's l2: 0.31972\n",
      "[190]\tvalid_0's auc: 0.982773\tvalid_0's l2: 0.319747\n",
      "[191]\tvalid_0's auc: 0.982783\tvalid_0's l2: 0.319851\n",
      "[192]\tvalid_0's auc: 0.982751\tvalid_0's l2: 0.319971\n",
      "[193]\tvalid_0's auc: 0.982685\tvalid_0's l2: 0.320043\n",
      "Early stopping, best iteration is:\n",
      "[188]\tvalid_0's auc: 0.982794\tvalid_0's l2: 0.319746\n",
      "Save model...\n",
      "Start predicting...\n",
      "error=0.664000\n"
     ]
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "# 加载你的数据\n",
    "# print('Load data...')\n",
    "# df_train = pd.read_csv('../regression/regression.train', header=None, sep='\\t')\n",
    "# df_test = pd.read_csv('../regression/regression.test', header=None, sep='\\t')\n",
    "#\n",
    "# y_train = df_train[0].values\n",
    "# y_test = df_test[0].values\n",
    "# X_train = df_train.drop(0, axis=1).values\n",
    "# X_test = df_test.drop(0, axis=1).values\n",
    "\n",
    "# 创建成lgb特征的数据集格式\n",
    "lgb_train = lgb.Dataset(X_train, y_train)  # 将数据保存到LightGBM二进制文件将使加载更快\n",
    "lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)  # 创建验证数据\n",
    "\n",
    "# 将参数写成字典下形式\n",
    "params = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',  # 设置提升类型\n",
    "    'objective': 'regression',  # 目标函数\n",
    "    'metric': {'l2', 'auc'},  # 评估函数\n",
    "    'num_leaves': 31,  # 叶子节点数\n",
    "    'learning_rate': 0.05,  # 学习速率\n",
    "    'feature_fraction': 0.9,  # 建树的特征选择比例\n",
    "    'bagging_fraction': 0.8,  # 建树的样本采样比例\n",
    "    'bagging_freq': 5,  # k 意味着每 k 次迭代执行bagging\n",
    "    'verbose': 1  # <0 显示致命的, =0 显示错误 (警告), >0 显示信息\n",
    "}\n",
    "\n",
    "print('Start training...')\n",
    "# 训练 cv and train\n",
    "gbm = lgb.train(params,\n",
    "                lgb_train,\n",
    "                num_boost_round=500,\n",
    "                valid_sets=lgb_eval,\n",
    "                early_stopping_rounds=5)  # 训练数据需要参数列表和数据集\n",
    "\n",
    "print('Save model...')\n",
    "\n",
    "gbm.save_model('model.txt')  # 训练后保存模型到文件\n",
    "\n",
    "print('Start predicting...')\n",
    "# 预测数据集\n",
    "y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration\n",
    "                     )  #如果在训练期间启用了早期停止，可以通过best_iteration方式从最佳迭代中获得预测\n",
    "# 评估模型\n",
    "print('error=%f' %\n",
    "      (sum(1\n",
    "           for i in range(len(y_pred)) if int(y_pred[i] > 0.5) != y_test[i]) /\n",
    "       float(len(y_pred))))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05ca2e65",
   "metadata": {},
   "source": [
    "## 2.scikit-learn接口"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3d7140ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.927\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "clf = LGBMClassifier(\n",
    "    boosting_type='gbdt',  # 提升树的类型 gbdt,dart,goss,rf\n",
    "    num_leaves=31,  #树的最大叶子数，对比xgboost一般为2^(max_depth)\n",
    "    max_depth=-1,  #最大树的深度\n",
    "    learning_rate=0.1,  #学习率\n",
    "    n_estimators=100,  # 拟合的树的棵树，相当于训练轮数\n",
    "    subsample_for_bin=200000,\n",
    "    objective=None,\n",
    "    class_weight=None,\n",
    "    min_split_gain=0.0,  # 最小分割增益\n",
    "    min_child_weight=0.001,  # 分支结点的最小权重\n",
    "    min_child_samples=20,\n",
    "    subsample=1.0,  # 训练样本采样率 行\n",
    "    subsample_freq=0,  # 子样本频率\n",
    "    colsample_bytree=1.0,  # 训练特征采样率 列\n",
    "    reg_alpha=0.0,  # L1正则化系数\n",
    "    reg_lambda=0.0,  # L2正则化系数\n",
    "    random_state=None,\n",
    "    n_jobs=-1,\n",
    "    silent=True,\n",
    ")\n",
    "clf.fit(X_train, y_train, eval_metric='auc')\n",
    "#设置验证集合 verbose=False不打印过程\n",
    "clf.fit(X_train, y_train)\n",
    "\n",
    "y_true, y_pred = y_test, clf.predict(X_test)\n",
    "print(\"Accuracy : %.4g\" % metrics.accuracy_score(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "911f1ef3",
   "metadata": {},
   "source": [
    "## 参考\n",
    "1.https://xgboost.readthedocs.io/\n",
    "\n",
    "2.https://lightgbm.readthedocs.io/\n",
    "\n",
    "3.https://blog.csdn.net/q383700092/article/details/53763328?locationNum=9&fps=1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
